{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from requests_html import HTMLSession\n",
    "import pprint\n",
    "from urllib.parse import urlparse, parse_qs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element 'dt' class=('search-title',)>, <Element 'dt' class=('search-title',)>, <Element 'dt' class=('search-title',)>, <Element 'dt' class=('search-title',)>, <Element 'dt' class=('search-title',)>]\n",
      "公司：\n",
      "行业：\n",
      "城市：\n",
      "薪资：\n",
      "更多：\n",
      "<Element 'dd' class=('comp-list',)>\n",
      "<Element 'dd' class=('short-dd', 'select-industry') data-param='industries'>\n",
      "<Element 'dd' data-param='city'>\n",
      "<Element 'dd' data-param='salary'>\n",
      "<Element 'dd' class=('dropdown', 'dropdown-time')>\n",
      "<Element 'dd' class=('dropdown', 'dropdown-jobkind')>\n",
      "<Element 'dd' class=('dropdown', 'dropdown-compscale')>\n",
      "<Element 'dd' class=('dropdown', 'dropdown-compkind')>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'中国500强': '/zhaopin/?init=-1&headckid=15bd2bdef1222146&flushckid=1&fromSearchBtn=2&keyword=python&compTag=155&ckid=15bd2bdef1222146&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=6d596adf7f5bcc150ae6779b10d3dc15&d_curPage=0&d_pageSize=40&d_headId=6d596adf7f5bcc150ae6779b10d3dc15',\n",
       " '2018互联网300强': '/zhaopin/?init=-1&headckid=15bd2bdef1222146&flushckid=1&fromSearchBtn=2&keyword=python&compTag=182&ckid=15bd2bdef1222146&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=6d596adf7f5bcc150ae6779b10d3dc15&d_curPage=0&d_pageSize=40&d_headId=6d596adf7f5bcc150ae6779b10d3dc15',\n",
       " '制造业500强': '/zhaopin/?init=-1&headckid=15bd2bdef1222146&flushckid=1&fromSearchBtn=2&keyword=python&compTag=186&ckid=15bd2bdef1222146&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=6d596adf7f5bcc150ae6779b10d3dc15&d_curPage=0&d_pageSize=40&d_headId=6d596adf7f5bcc150ae6779b10d3dc15',\n",
       " 'AI创新成长50强 ': '/zhaopin/?init=-1&headckid=15bd2bdef1222146&flushckid=1&fromSearchBtn=2&keyword=python&compTag=189&ckid=15bd2bdef1222146&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=6d596adf7f5bcc150ae6779b10d3dc15&d_curPage=0&d_pageSize=40&d_headId=6d596adf7f5bcc150ae6779b10d3dc15',\n",
       " '独角兽': '/zhaopin/?init=-1&headckid=15bd2bdef1222146&flushckid=1&fromSearchBtn=2&keyword=python&compTag=130&ckid=15bd2bdef1222146&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=6d596adf7f5bcc150ae6779b10d3dc15&d_curPage=0&d_pageSize=40&d_headId=6d596adf7f5bcc150ae6779b10d3dc15',\n",
       " '上市公司': '/zhaopin/?init=-1&headckid=15bd2bdef1222146&flushckid=1&fromSearchBtn=2&keyword=python&compTag=156&ckid=15bd2bdef1222146&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=6d596adf7f5bcc150ae6779b10d3dc15&d_curPage=0&d_pageSize=40&d_headId=6d596adf7f5bcc150ae6779b10d3dc15'}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from requests_html import HTMLSession\n",
    "url = \"https://www.liepin.com/zhaopin/?keyword=python\"\n",
    "session = HTMLSession()\n",
    "r = session.get( url )\n",
    "主要元素 = r.html.xpath('//div[@data-selector=\"search-conditions\"]')\n",
    "list_search_title = 主要元素[0].xpath('//dt[@class=\"search-title\"]')\n",
    "print (主要元素[0].xpath('//dt[@class=\"search-title\"]'))\n",
    "for x in list_search_title:\n",
    "    print (x.text)\n",
    "list_search_dd = 主要元素[0].xpath('//dt[@class=\"search-title\"]/following-sibling::dd')\n",
    "for x in list_search_dd:\n",
    "    print (x)\n",
    "行业数据选择器链结 = r.html.xpath('//div[contains(@class,\"hot-comp-tags\")]/a')\n",
    "               \n",
    "行业数据选择器链结 = { x.xpath(\"a/text()\")[0]:x.xpath(\"a/@href\")[0] for x in 行业数据选择器链结}\n",
    "行业数据选择器链结"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def requests_liepin( url, params):\n",
    "    r = session.get( url , params = payload)\n",
    "\n",
    "    # 先取特定元素, 精准打击其子后辈\n",
    "    主要元素 = r.html.xpath( '//ul[@class=\"sojob-list\"]/li')\n",
    "\n",
    "    # 作为xpath字典，键为我要抓的牛肉名称，值为xpath\n",
    "    dict_xpaths={ \n",
    "        'text': {\n",
    "            'edu':      '//div[contains(@class,\"job-info\")]/p/span[@class=\"edu\"]',\n",
    "            '经验':      '//div[contains(@class,\"job-info\")]/p/span[@class=\"edu\"]/following-sibling::span',\n",
    "            '薪水':    '//div[contains(@class,\"job-info\")]/p/span[@class=\"text-warning\"]', \n",
    "            '时间':    '//div[contains(@class,\"job-info\")]/p/time/@title', \n",
    "            '职称':    '//div[contains(@class,\"job-info\")]/h3/a', \n",
    "            '公司地点': '//div[contains(@class,\"job-info\")]/p/a',\n",
    "            '公司名称': '//div[contains(@class,\"sojob-item-main\")]//p[@class=\"company-name\"]/a', \n",
    "        },\n",
    "        'text_content': {\n",
    "        },\n",
    "        'href': {\n",
    "            '链结':    '//div[contains(@class,\"job-info\")]/h3/a', \n",
    "            '公司URL': '//div[contains(@class,\"sojob-item-main\")]//p[@class=\"company-name\"]/a', \n",
    "        }\n",
    "    }\n",
    "\n",
    "    def get_e_text_content(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [e.xpath(_xpath_)[0].lxml.text_content() for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    def get_e_text(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [\"\".join([x.strip() if type(x) is str else x.text.strip() for x in e.xpath(_xpath_)]) for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    def get_e_href(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [list(e.xpath(_xpath_, first=True).absolute_links)[0] \\\n",
    "                   if len(e.xpath(_xpath_, first=True).absolute_links) >= 1  \\\n",
    "                   else \"\" for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    # 只对主要元素下进行.xpath取值\n",
    "    数据字典 = dict()\n",
    "\n",
    "    数据字典 = {k:get_e_text_content(v) for k,v in dict_xpaths['text_content'].items()}\n",
    "    数据字典.update({k:get_e_text(v) for k,v in dict_xpaths['text'].items()})\n",
    "    数据字典.update({k:get_e_href(v) for k,v in dict_xpaths['href'].items()})\n",
    "\n",
    "    数据 = pd.DataFrame(数据字典)\n",
    "    #数据.to_excel(\"20春_Web数据挖掘_week03_liepin.xlsx\", sheet_name=\"搜查结果\")\n",
    "    return (数据)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'中国500强': '155', '2018互联网300强': '182', '制造业500强': '186', 'AI创新成长50强 ': '189', '独角兽': '130', '上市公司': '156'}\n"
     ]
    }
   ],
   "source": [
    "def parse_url_qs_for_compTag (url):\n",
    "    six_parts = urlparse(url) \n",
    "    out = parse_qs(six_parts.query)\n",
    "    return (out)\n",
    "\n",
    "参数模板 = parse_url_qs_for_compTag(list(行业数据选择器链结.values())[0])\n",
    "#print(参数模板)\n",
    "#display([ parse_url_qs_for_compTag(x)['industryType'][0] for x in 行业数据选择器链结.values()])\n",
    "\n",
    "字典_compTag = { k:parse_url_qs_for_compTag(v)['compTag'][0] for k,v in 行业数据选择器链结.items()}\n",
    "print (字典_compTag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'init': ['-1'],\n",
       " 'headckid': ['15bd2bdef1222146'],\n",
       " 'flushckid': ['1'],\n",
       " 'fromSearchBtn': ['2'],\n",
       " 'keyword': ['python'],\n",
       " 'compTag': ['155'],\n",
       " 'ckid': ['15bd2bdef1222146'],\n",
       " 'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'],\n",
       " 'd_sfrom': ['search_unknown'],\n",
       " 'd_ckId': ['6d596adf7f5bcc150ae6779b10d3dc15'],\n",
       " 'd_curPage': ['0'],\n",
       " 'd_pageSize': ['40'],\n",
       " 'd_headId': ['6d596adf7f5bcc150ae6779b10d3dc15']}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "参数模板"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "155\n",
      "155\n",
      "产品经理\n",
      "产品经理 10\n",
      "155\n",
      "python\n",
      "python 10\n",
      "182\n",
      "182\n",
      "产品经理\n",
      "产品经理 10\n",
      "182\n",
      "python\n",
      "python 10\n",
      "186\n",
      "186\n",
      "产品经理\n",
      "产品经理 10\n",
      "186\n",
      "python\n",
      "python 10\n",
      "189\n",
      "189\n",
      "产品经理\n",
      "产品经理 10\n",
      "189\n",
      "python\n",
      "python 10\n",
      "130\n",
      "130\n",
      "产品经理\n",
      "产品经理 10\n",
      "130\n",
      "python\n",
      "python 10\n",
      "156\n",
      "156\n",
      "产品经理\n",
      "产品经理 10\n",
      "156\n",
      "python\n",
      "python 10\n",
      "Wall time: 11min 16s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# B-3 多个页面+多个关键词\n",
    "import time\n",
    "from random import random\n",
    "def 参数模板生成(keyword,curPage,compTag):\n",
    "    参数 = 参数模板.copy()\n",
    "    参数['curPage'] = curPage\n",
    "    参数['keyword'] = keyword\n",
    "    参数['compTag'] = compTag    \n",
    "    return (参数)\n",
    "time.sleep(3+4*random())\n",
    "url = \"https://www.liepin.com/zhaopin/\"\n",
    "xpath_翻页a = '//div[@class=\"pagerbar\"]/a[starts-with(@href,\"/zhaopin\")]'\n",
    "keywords = ['产品经理','python']\n",
    "compTags = ['155','182','186','189','130','156']\n",
    "list_df = list()\n",
    "\n",
    "## 第一页试探有多长的页面\n",
    "for comp in compTags:\n",
    "    print(comp)\n",
    "    for key in keywords:\n",
    "        print(comp)\n",
    "        print(key)\n",
    "        payload = 参数模板生成(keyword=[key],curPage=['0'],compTag=[comp])\n",
    "        # requests_liepin为上面定义的函数模块，返回df格式的数据表格\n",
    "        df = requests_liepin( url, params = payload)\n",
    "        href_列表 = [x.xpath('//@href')[0] for x in r.html.xpath(xpath_翻页a)]\n",
    "        df = pd.DataFrame([ urlparse(x) for x in href_列表])\n",
    "        df_qs = pd.DataFrame([{k:v[0] for k,v in parse_qs(x).items()} for x in df['query'] ])\n",
    "        df_qs = df_qs.assign (curPage_int=df_qs.curPage.astype(int)) # 变成整数\n",
    "        长度 = df_qs.curPage_int.max()+1\n",
    "        参数_keyword_X_curPage = { \n",
    "            i:参数模板生成(curPage = [i], \\\n",
    "                          keyword = [key],compTag=[comp]) \\\n",
    "            for i in range(0,长度)\\\n",
    "            }\n",
    "        #print (参数_keyword_X_curPage)\n",
    "        print (key,长度)\n",
    "\n",
    "        for k,v in 参数_keyword_X_curPage.items():\n",
    "            payload = v\n",
    "            df = requests_liepin( url, params = payload)\n",
    "            time.sleep(3+4*random())  #放慢脚步 3-7秒, 平均约5秒\n",
    "            df = df.assign (keyword = key)  # 区分  keyword    \n",
    "            df = df.assign (curPage = k)  # 区分  curPage\n",
    "            df = df.assign (compTag = comp)\n",
    "            list_df.append(df)\n",
    "        \n",
    "df_all = pd.concat(list_df).reset_index()\n",
    "df_all.index.name = '序'\n",
    "\n",
    "df_all.to_excel(\"python和web_AI创新成长50强500强猎聘企业.xlsx\",\\\n",
    "                sheet_name=\"_\".join(keywords))\n",
    "# 预估时间: 2*5秒*10 =100\n",
    "# 预估数量: 2*40*10 =800"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
